# 分词器
from typing import Dict
import random
from tqdm import tqdm
from transformers import AutoTokenizer
import json
from datasets import load_dataset
from tokenizers import (
    decoders,
    models,
    normalizers,
    pre_tokenizers,
    processors,
    trainers,
    Tokenizer,
)
import os

class NanodsTokenizer(object):
    def __init__(self):
        self.name = 'nanods.nanods_tokenizer.NanodsTokenizer'

    @staticmethod
    def train_tokenizer(params:Dict = {}) -> None:
        print(f'分词器训练程序 v0.0.1')
        # 测试读取数据
        data_path = './datasets/tokenizer_train.jsonl'
        texts = NanodsTokenizer.read_texts_from_jsonl(data_path)
        # 打印前几行文本
        for i, text in enumerate(texts):
            if i < 5:
                print(text)
            else:
                break
        # 初始化tokenizer
        tokenizer = Tokenizer(models.BPE())
        tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
        # 定义特殊token
        special_tokens = ["<unk>", "<s>", "</s>"]
        # 设置训练器并添加特殊token
        trainer = trainers.BpeTrainer(
            vocab_size=6400,
            special_tokens=special_tokens,  # 确保这三个token被包含
            show_progress=True,
            initial_alphabet=pre_tokenizers.ByteLevel.alphabet()
        )
        print("分词器初始化成功，准备训练。")
        # 读取文本数据
        texts = NanodsTokenizer.read_texts_from_jsonl(data_path)
        # 训练tokenizer
        tokenizer.train_from_iterator(texts, trainer=trainer)
        print("分词器训练完成！")
        # 设置解码器
        tokenizer.decoder = decoders.ByteLevel()
        # 保存tokenizer
        tokenizer_dir = "work/model/miniDeepSeek_tokenizer"
        os.makedirs(tokenizer_dir, exist_ok=True)
        tokenizer.save(os.path.join(tokenizer_dir, "tokenizer.json"))
        tokenizer.model.save("work/model/miniDeepSeek_tokenizer")
        # 手动创建配置文件
        config = {
            "add_bos_token": False,
            "add_eos_token": False,
            "add_prefix_space": True,
            "added_tokens_decoder": {
                "0": {
                    "content": "<unk>",
                    "lstrip": False,
                    "normalized": False,
                    "rstrip": False,
                    "single_word": False,
                    "special": True
                    },
                "1": {
                    "content": "<s>",
                    "lstrip": False,
                    "normalized": False,
                    "rstrip": False,
                    "single_word": False,
                    "special": True
                    },
                "2": {
                    "content": "</s>",
                    "lstrip": False,
                    "normalized": False,
                    "rstrip": False,
                    "single_word": False,
                    "special": True
                    }
            },
            "bos_token": "<s>",
            "clean_up_tokenization_spaces": False,
            "eos_token": "</s>",
            "legacy": True,
            "model_max_length": 1000000000000000019884624838656,
            "pad_token": None,
            "sp_model_kwargs": {},
            "spaces_between_special_tokens": False,
            "tokenizer_class": "PreTrainedTokenizerFast",
            "unk_token": "<unk>",
            "use_default_system_prompt": False,
            "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<s>user\\n' + content + '</s>\\n<s>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' + '\\n' }}{% endif %}{% endfor %}"
        }
        # 保存配置文件
        with open(os.path.join(tokenizer_dir, "tokenizer_config.json"), "w", encoding="utf-8") as config_file:
            json.dump(config, config_file, ensure_ascii=False, indent=4)
        print("Tokenizer 保存成功！")

    @staticmethod
    def test_tokenizer(params:Dict = {}) -> None:
        print(f'测试分词器应用 v0.0.1')
        # 加载预训练的tokenizer
        tokenizer = AutoTokenizer.from_pretrained("./work/model/miniDeepSeek_tokenizer")
        # 测试一段对话
        messages = [
            {"role": "system", "content": "你是一个优秀的聊天机器人，总是给我正确的回应！"},
            {"role": "user", "content": '是椭圆形的'},
            {"role": "assistant", "content": '456'},
            {"role": "user", "content": '456'},
            {"role": "assistant", "content": '789'}
        ]
        # 使用模板进行文本处理
        new_prompt = tokenizer.apply_chat_template(messages, tokenize=True)
        print(new_prompt)





    @staticmethod
    def read_texts_from_jsonl(file_path):
        with open(file_path, 'r', encoding='utf-8') as f:
            for line in f:
                data = json.loads(line)
                yield data['text']